# -*- coding: utf-8 -*-

import requests

import openpyxl

import re

from bs4 import BeautifulSoup

from multiprocessing.dummy import Pool as ThreadPool

wb = openpyxl.load_workbook('company_database.xlsx')

ws = wb.active

start = 10000

f = 'F'

urlstr = 'https://tieba.baidu.com/p/4133096680'

query = '/q/'

def getEmail(url):

print('grab email....')

html = requests.get(url[0])

soup = BeautifulSoup(html.text,"lxml")

pudge = soup.find_all("p", text="Email")

if pudge:

email = re.findall(r"[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+",html.text,re.I)[0]

return email,url[1]

else:

return '',url[1]

def writeExcel(mailCount):

ws['L'+str(mailCount[1])].value = mailCount[0]

print('%d save ok: ' % mailCount[1]+mailCount[0])

def geturl(num):

print('Grabing url')

celstr = f + str(num)

compname = ws[celstr].value

url = urlstr + compname.replace(' ', '+') + query

return url,num

def main(start):

pool1 = ThreadPool(16)

urlCount = pool1.map(geturl, range(start,17722))

pool1.close()

pool1.join()

pool2 = ThreadPool(16)

mailCount = pool2.map(getEmail,urlCount)

pool2.close()

pool2.join()

pool3 = ThreadPool(16)

pool3.map(writeExcel,mailCount)

pool3.close()

pool3.join()

wb.save('company_database.xlsx')

print (' ok!')

if __name__ == '__main__':

main(start)